{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('../')\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from matplotlib import pyplot as plt\n",
    "from tqdm import tqdm_notebook\n",
    "import pickle\n",
    "import os\n",
    "import logging\n",
    "import time\n",
    "from IPython.core.debugger import set_trace\n",
    "import glob\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "from utils.utils import save_checkpoint, load_checkpoint, set_logger\n",
    "from utils.gpu_utils import set_n_get_device\n",
    "\n",
    "from dataset.dataset import prepare_trainset\n",
    "\n",
    "#from model.model_unet_classify_zero import UNetResNet34\n",
    "#from model.model_unet_classify_zero2 import UNetResNet34\n",
    "from model.efficientnet_pytorch.efficientnet_pytorch import EfficientNet\n",
    "\n",
    "from model.deeplab_model_kaggler.lr_scheduler import LR_Scheduler\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====MODEL ACHITECTURE: efficientnet-b5====\n"
     ]
    }
   ],
   "source": [
    "######### Config the training process #########\n",
    "#device = set_n_get_device(\"0, 1, 2, 3\", data_device_id=\"cuda:0\")#0, 1, 2, 3, IMPORTANT: data_device_id is set to free gpu for storing the model, e.g.\"cuda:1\"\n",
    "MODEL = 'efficientnet-b5' #'b3','b5'\n",
    "print('====MODEL ACHITECTURE: %s===='%MODEL)\n",
    "\n",
    "device = set_n_get_device(\"0\", data_device_id=\"cuda:0\")#0, 1, 2, 3, IMPORTANT: data_device_id is set to free gpu for storing the model, e.g.\"cuda:1\"\n",
    "multi_gpu = None #[0, 1] #None\n",
    "\n",
    "SEED = 2008\n",
    "debug = False# if True, load 100 samples\n",
    "IMG_SIZE = (256,1600)\n",
    "BATCH_SIZE = 8\n",
    "NUM_WORKERS = 24\n",
    "torch.cuda.manual_seed_all(SEED)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Count images in train/test folder:  5546 3698\n",
      "Count of trainset (for training):  4714\n",
      "Count of validset (for training):  832\n"
     ]
    }
   ],
   "source": [
    "train_dl, val_dl = prepare_trainset(BATCH_SIZE, NUM_WORKERS, SEED, IMG_SIZE, debug)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i, (images, masks) in enumerate(train_dl):\n",
    "    images = images.to(device=device, dtype=torch.float)\n",
    "    #1 for non-zero-mask\n",
    "    truth = (torch.sum(masks.reshape(masks.size()[0], masks.size()[1], -1), dim=2, keepdim=False)!=0).to(device=device, dtype=torch.float)\n",
    "    if i==0:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([8, 1, 256, 1600]), torch.Size([8, 4]))"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "images.size(), truth.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#truth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "../model/efficientnet-b5-586e6cc6.pth\n",
      "====Loading pretrained weights done====\n"
     ]
    }
   ],
   "source": [
    "net = EfficientNet.from_name(MODEL, debug=debug)#override_params={'num_classes': 1}\n",
    "net.load_state_dict(torch.load(glob.glob('../model/%s*'%MODEL)[0]))\n",
    "in_features = net._fc.in_features\n",
    "net._fc = nn.Linear(in_features, 4)#num_classes=1\n",
    "net = net.cuda(device=device)\n",
    "print(glob.glob('../model/%s*'%MODEL)[0])\n",
    "print('====Loading pretrained weights done====')\n",
    "\n",
    "if multi_gpu is not None:\n",
    "    net = nn.DataParallel(net, device_ids=multi_gpu)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "logit = net(images)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 4])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logit.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(1.9995, device='cuda:0', grad_fn=<MultilabelMarginLossBackward>)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Loss_FUNC = nn.MultiLabelMarginLoss()\n",
    "# loss = Loss_FUNC(torch.sigmoid(logit), truth.long())\n",
    "# loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.7063, device='cuda:0', grad_fn=<BinaryCrossEntropyWithLogitsBackward>)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "if multi_gpu:\n",
    "    loss = net.module.criterion(logit, truth)\n",
    "else:\n",
    "    loss = net.criterion(logit, truth)\n",
    "\n",
    "loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.4037, 11, 16, 3, 2, 0.5625)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "if multi_gpu:\n",
    "    _, metric, tn, fp, fn, tp, pos_percent = net.module.metric(logit, truth)\n",
    "else:\n",
    "    _, metric, tn, fp, fn, tp, pos_percent = net.metric(logit, truth)\n",
    "\n",
    "metric, tn, fp, fn, tp, pos_percent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "#label_df = pd.read_csv('data/raw/NIH/NIH external data/label_df_pneumothorax.csv').set_index('img_id')\n",
    "\n",
    "for idx, img_id in enumerate(label_df.index):\n",
    "    if idx>10000:\n",
    "        break\n",
    "    has_pneumothorax = label_df.loc[img_id, 'has_pneumothorax']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for idx, f in enumerate(glob.glob('data/raw/NIH/NIH external data/images*/*')):\n",
    "    img = plt.imread(f)\n",
    "    if len(img.shape)==3:\n",
    "        print(img.shape)\n",
    "        break\n",
    "    if idx>100:\n",
    "        break\n",
    "#img_path = glob.glob('data/raw/NIH/NIH external data/images*/00000001_000.png')[0]#00000003_000.png\n",
    "#img = plt.imread(img_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "img.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#plt.imshow(img)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## predict the validset, and analyse  \n",
    "**first, predict zero-nonzero-mask**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#move checkpoint from gamma machine to here\n",
    "cd checkpoint\n",
    "scp -r endi.niu@10.171.36.214:/home/endi.niu/SIIM/checkpoint/nonzero_classifier_UNetResNet34_768_v1_seed1234/ nonzero_classifier_UNetResNet34_768_v1_seed1234\n",
    "cd logging\n",
    "scp -r endi.niu@10.171.36.214:/home/endi.niu/SIIM/logging/nonzero_classifier_UNetResNet34_768_v1_seed1234.log nonzero_classifier_UNetResNet34_768_v1_seed1234.log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('../')\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import math\n",
    "from matplotlib import pyplot as plt\n",
    "from tqdm import tqdm_notebook\n",
    "import pickle\n",
    "import os\n",
    "import logging\n",
    "import time\n",
    "from IPython.core.debugger import set_trace\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "from utils.utils import save_checkpoint, load_checkpoint, set_logger\n",
    "from utils.gpu_utils import set_n_get_device\n",
    "\n",
    "#from model.model_unet_classify_zero import UNetResNet34, predict_proba\n",
    "from model.efficientnet_pytorch.efficientnet_pytorch.model import EfficientNet, predict_proba\n",
    "\n",
    "from dataset.dataset import prepare_trainset\n",
    "\n",
    "from sklearn.metrics import roc_auc_score, confusion_matrix, classification_report\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====MODEL ACHITECTURE: efficientnet-b5====\n"
     ]
    }
   ],
   "source": [
    "######### Config the training process #########\n",
    "#device = set_n_get_device(\"0, 1, 2, 3\", data_device_id=\"cuda:0\")#0, 1, 2, 3, IMPORTANT: data_device_id is set to free gpu for storing the model, e.g.\"cuda:1\"\n",
    "MODEL = 'efficientnet-b5' #'b3','b5'\n",
    "#AUX_LOGITS = True#False, only for 'INCEPTION_V3'\n",
    "print('====MODEL ACHITECTURE: %s===='%MODEL)\n",
    "\n",
    "device = set_n_get_device(\"0,1\", data_device_id=\"cuda:0\")#0, 1, 2, 3, IMPORTANT: data_device_id is set to free gpu for storing the model, e.g.\"cuda:1\"\n",
    "multi_gpu = [0, 1]#use 2 gpus\n",
    "\n",
    "SEED = 2024\n",
    "debug = False # if True, load 100 samples\n",
    "IMG_SIZE = (512, 768)\n",
    "BATCH_SIZE = 32\n",
    "NUM_WORKERS = 24"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Count images in train/test folder:  5546 3698\n",
      "Count of trainset (for training):  4714\n",
      "Count of validset (for training):  832\n"
     ]
    }
   ],
   "source": [
    "train_dl, val_dl = prepare_trainset(BATCH_SIZE, NUM_WORKERS, SEED, IMG_SIZE, debug)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(832, 4)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# y should be makeup\n",
    "y_valid = []\n",
    "for i, (images, masks) in enumerate(val_dl):\n",
    "    #if i==10:\n",
    "    #    break\n",
    "    #truth = masks.to(device=device, dtype=torch.float)\n",
    "    truth = (torch.sum(masks.reshape(masks.size()[0], masks.size()[1], -1), dim=2, keepdim=False)!=0)#.to(device=device, dtype=torch.float)\n",
    "    y_valid.append(truth.numpy())\n",
    "y_valid = np.concatenate(y_valid, axis=0)\n",
    "y_valid.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5336538461538461"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_valid.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "use TTA\n",
      "Complete  ../checkpoint/nonzero_classifier_efficientnet-b5_512x768_v4_seed2025/best.pth.tar\n",
      "use TTA\n",
      "Complete  ../checkpoint/nonzero_classifier_efficientnet-b5_512x768_v4_seed2026/best.pth.tar\n",
      "use TTA\n",
      "Complete  ../checkpoint/nonzero_classifier_efficientnet-b5_512x768_v4_seed2027/best.pth.tar\n",
      "use TTA\n",
      "Complete  ../checkpoint/nonzero_classifier_efficientnet-b5_512x768_v4_seed2028/best.pth.tar\n",
      "use TTA\n",
      "Complete  ../checkpoint/nonzero_classifier_efficientnet-b5_512x768_v4_seed2029/best.pth.tar\n",
      "CPU times: user 19min 46s, sys: 1min 27s, total: 21min 14s\n",
      "Wall time: 4min 56s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "ensemble = True\n",
    "\n",
    "#### multiple fold ensemble sigmoids\n",
    "if ensemble:\n",
    "    checkpoint_path_list = ['../checkpoint/nonzero_classifier_efficientnet-b5_512x768_v4_seed2025/best.pth.tar', \n",
    "                            '../checkpoint/nonzero_classifier_efficientnet-b5_512x768_v4_seed2026/best.pth.tar', \n",
    "                            '../checkpoint/nonzero_classifier_efficientnet-b5_512x768_v4_seed2027/best.pth.tar', \n",
    "                            '../checkpoint/nonzero_classifier_efficientnet-b5_512x768_v4_seed2028/best.pth.tar', \n",
    "                            '../checkpoint/nonzero_classifier_efficientnet-b5_512x768_v4_seed2029/best.pth.tar']\n",
    "    \n",
    "    preds_valid = None\n",
    "    for checkpoint_path in checkpoint_path_list:\n",
    "        net = EfficientNet.from_name(MODEL, debug=debug)#override_params={'num_classes': 1}\n",
    "        in_features = net._fc.in_features\n",
    "        net._fc = nn.Linear(in_features, 4)#num_classes=1\n",
    "        net = net.cuda(device=device)\n",
    "        \n",
    "        net, _ = load_checkpoint(checkpoint_path, net)\n",
    "        \n",
    "        if multi_gpu is not None:\n",
    "            net = nn.DataParallel(net, device_ids=multi_gpu)\n",
    "\n",
    "        _preds_valid = predict_proba(net, val_dl, device, multi_gpu=multi_gpu, mode='valid', tta=True)\n",
    "        if preds_valid is None:\n",
    "            preds_valid = _preds_valid\n",
    "        else:\n",
    "            preds_valid += _preds_valid\n",
    "        preds_valid /= len(checkpoint_path_list)\n",
    "        print('Complete ', checkpoint_path)\n",
    "\n",
    "#### single fold\n",
    "else:\n",
    "    checkpoint_path = '../checkpoint/nonzero_classifier_efficientnet-b5_512x768_v4_seed2025/best.pth.tar'\n",
    "\n",
    "    net = EfficientNet.from_name(MODEL, debug=debug)#override_params={'num_classes': 1}\n",
    "    in_features = net._fc.in_features\n",
    "    net._fc = nn.Linear(in_features, 4)#num_classes=1\n",
    "    net = net.cuda(device=device)\n",
    "\n",
    "    net, _ = load_checkpoint(checkpoint_path, net)\n",
    "\n",
    "    if multi_gpu is not None:\n",
    "        net = nn.DataParallel(net, device_ids=multi_gpu)\n",
    "\n",
    "    preds_valid = predict_proba(net, val_dl, device, multi_gpu=multi_gpu, mode='valid', tta=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((832, 4), (832, 4))"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_valid.shape, preds_valid.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_auc_score, confusion_matrix\n",
    "import copy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigmoid(x):\n",
    "    return 1 / (1 + np.exp(-x))\n",
    "\n",
    "def cal_metric(logit, truth):\n",
    "    #pred = sigmoid(logit.cpu().detach())\n",
    "    pred = sigmoid(logit)\n",
    "    #truth = truth.cpu().detach().numpy()\n",
    "    ##\n",
    "    THRESHOLD_candidate = np.arange(0.01, 0.99, 0.01)\n",
    "    N = len(THRESHOLD_candidate)\n",
    "    best_threshold = [0.5, 0.5, 0.5, 0.5]\n",
    "    best_score = -1\n",
    "    tn, fp, fn, tp, pos_percent = 0, 0, 0, 0, 0.0\n",
    "\n",
    "    for ch in range(4):\n",
    "        for i in range(N):\n",
    "            THRESHOLD = copy.deepcopy(best_threshold)\n",
    "            THRESHOLD[ch] = THRESHOLD_candidate[i]\n",
    "            _pred = pred>THRESHOLD\n",
    "            _pred, truth = _pred.reshape(-1, 1), truth.reshape(-1, 1)\n",
    "            \n",
    "            _tn, _fp, _fn, _tp = confusion_matrix(truth, _pred).ravel()\n",
    "            _auc = round(roc_auc_score(truth, _pred), 5)\n",
    "            if _tn+_fn==0:\n",
    "                fn_rate = 9999\n",
    "            else:\n",
    "                fn_rate = round(_fn/(_tn+_fn), 5)\n",
    "            _pos_percent = (_tp+_fp)/(_tp+_fp+_tn+_fn)\n",
    "            \n",
    "            if _auc > best_score:\n",
    "                best_threshold = copy.deepcopy(THRESHOLD)\n",
    "                best_score = _auc\n",
    "                tn, fp, fn, tp, pos_percent = _tn, _fp, _fn, _tp, _pos_percent\n",
    "    return best_threshold, best_score, tn, fp, fn, tp, pos_percent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([0.49, 0.49, 0.49, 0.49], 0.80888, 1255, 297, 339, 1437, 0.5210336538461539)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_threshold, best_score, tn, fp, fn, tp, pos_percent = cal_metric(preds_valid, y_valid)\n",
    "best_threshold, best_score, tn, fp, fn, tp, pos_percent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.80      0.74      0.77       424\n",
      "           1       0.84      0.87      0.86       383\n",
      "           2       0.78      0.74      0.76       399\n",
      "           3       0.86      0.86      0.86       570\n",
      "\n",
      "   micro avg       0.83      0.81      0.82      1776\n",
      "   macro avg       0.82      0.81      0.81      1776\n",
      "weighted avg       0.83      0.81      0.82      1776\n",
      " samples avg       0.85      0.84      0.81      1776\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/endi.niu/anaconda3/lib/python3.7/site-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_valid, sigmoid(preds_valid)>best_threshold))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.80      0.62      0.70       434\n",
      "           1       0.84      0.79      0.82       359\n",
      "           2       0.82      0.57      0.67       424\n",
      "           3       0.81      0.84      0.82       559\n",
      "\n",
      "   micro avg       0.82      0.71      0.76      1776\n",
      "   macro avg       0.82      0.71      0.75      1776\n",
      "weighted avg       0.82      0.71      0.76      1776\n",
      " samples avg       0.82      0.74      0.74      1776\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/endi.niu/anaconda3/lib/python3.7/site-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in samples with no predicted labels.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_valid, sigmoid(preds_valid)>best_threshold))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigmoid(x):\n",
    "    return 1 / (1 + np.exp(-x))\n",
    "\n",
    "def predict_mask(logit, best_threshold):\n",
    "    \"\"\"Transform each prediction into mask.\n",
    "    input shape: (256, 256)\n",
    "    \"\"\"\n",
    "    #pred mask 0-1 pixel-wise\n",
    "    #n = logit.shape[0]\n",
    "    #IMG_SIZE = logit.shape[-1] #256\n",
    "    #EMPTY_THRESHOLD = 100.0*(IMG_SIZE/128.0)**2 #count of predicted mask pixles<threshold, predict as empty mask image\n",
    "    #MASK_THRESHOLD = 0.22\n",
    "    #logit = torch.sigmoid(torch.from_numpy(logit)).view(n, -1)\n",
    "    #pred = (logit>MASK_THRESHOLD).long()\n",
    "    #pred[pred.sum(dim=1) < EMPTY_THRESHOLD, ] = 0 #bug here, found it, the bug is input shape is (256, 256) not (16,256,256)\n",
    "    logit = sigmoid(logit)#.reshape(n, -1)\n",
    "    pred = (logit>best_threshold).astype(np.int)\n",
    "    return pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## predict the testset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "import glob\n",
    "from dataset.dataset import prepare_testset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3698, '10055ff')"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_fnames = [f.split('/')[-1][:-4] for f in glob.glob('../data/raw/test/*')]\n",
    "len(test_fnames), test_fnames[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_dl = prepare_testset(BATCH_SIZE, NUM_WORKERS, IMG_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "use TTA\n",
      "use TTA\n",
      "use TTA\n",
      "use TTA\n",
      "use TTA\n",
      "CPU times: user 1h 24min 29s, sys: 5min 21s, total: 1h 29min 51s\n",
      "Wall time: 16min 24s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "if ensemble:\n",
    "    preds_test = None\n",
    "    for checkpoint_path in checkpoint_path_list:\n",
    "        net = EfficientNet.from_name(MODEL, debug=debug)#override_params={'num_classes': 1}\n",
    "        in_features = net._fc.in_features\n",
    "        net._fc = nn.Linear(in_features, 4)#num_classes=1\n",
    "        net = net.cuda(device=device)\n",
    "        \n",
    "        net, _ = load_checkpoint(checkpoint_path, net)\n",
    "        \n",
    "        if multi_gpu is not None:\n",
    "            net = nn.DataParallel(net, device_ids=multi_gpu)\n",
    "        \n",
    "        _preds_test = predict_proba(net, test_dl, device, multi_gpu=multi_gpu, mode='test', tta=True)\n",
    "        \n",
    "        if preds_test is None:\n",
    "            preds_test = _preds_test\n",
    "        else:\n",
    "            preds_test += _preds_test\n",
    "        preds_test /= len(checkpoint_path_list)\n",
    "\n",
    "else:\n",
    "    preds_test = predict_proba(net, test_dl, device, multi_gpu=multi_gpu, mode='test', tta=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3698, 4)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## build submission (for empty/non-empty)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_threshold = [0.51, 0.51, 0.51, 0.51]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.08883731, -0.16244433,  0.11221843, ...,  0.0632857 ,\n",
       "       -0.04195655, -0.0703424 ])"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds_clf = (sigmoid(preds_test)-best_threshold).astype(np.float).reshape(1,-1)[0]\n",
    "preds_clf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "14792\n"
     ]
    }
   ],
   "source": [
    "img_id_ch = []\n",
    "for fname in test_fnames:\n",
    "    for name in ['Fish', 'Flower', 'Gravel', 'Sugar']:\n",
    "        img_id_ch.append(fname+'.jpg_%s'%name)\n",
    "\n",
    "sub_df = pd.DataFrame({'Image_Label': img_id_ch, 'has_mask': preds_clf})\n",
    "print(len(sub_df.index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Image_Label</th>\n",
       "      <th>has_mask</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10055ff.jpg_Fish</td>\n",
       "      <td>-0.088837</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10055ff.jpg_Flower</td>\n",
       "      <td>-0.162444</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>10055ff.jpg_Gravel</td>\n",
       "      <td>0.112218</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          Image_Label  has_mask\n",
       "0    10055ff.jpg_Fish -0.088837\n",
       "1  10055ff.jpg_Flower -0.162444\n",
       "2  10055ff.jpg_Gravel  0.112218"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub_df.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### use clf on segment model output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Image_Label</th>\n",
       "      <th>EncodedPixels</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10055ff.jpg_Fish</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10055ff.jpg_Flower</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>10055ff.jpg_Gravel</td>\n",
       "      <td>1217 12 1560 2 1564 20 1599 2 1618 4 1899 3 19...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>10055ff.jpg_Sugar</td>\n",
       "      <td>23 100 266 37 311 5 360 127 592 81 706 135 939...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6e54759.jpg_Fish</td>\n",
       "      <td>1829 6 2178 7 2527 8 2876 10 2900 1 3226 10 32...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          Image_Label                                      EncodedPixels\n",
       "0    10055ff.jpg_Fish                                                   \n",
       "1  10055ff.jpg_Flower                                                   \n",
       "2  10055ff.jpg_Gravel  1217 12 1560 2 1564 20 1599 2 1618 4 1899 3 19...\n",
       "3   10055ff.jpg_Sugar  23 100 266 37 311 5 360 127 592 81 706 135 939...\n",
       "4    6e54759.jpg_Fish  1829 6 2178 7 2527 8 2876 10 2900 1 3226 10 32..."
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "seg_df = pd.read_csv('../submission/1110_deeplabv3plus_resnet_512x768_v6_seed2001.csv.gz').fillna('')\n",
    "seg_df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6094510546241212"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(seg_df['EncodedPixels']!='').mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "seg_df = seg_df.merge(sub_df, on=['Image_Label'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "for idx in seg_df.index:\n",
    "    has_mask = seg_df.loc[idx, 'has_mask']>0\n",
    "    if not has_mask:\n",
    "        seg_df.loc[idx, 'EncodedPixels'] = ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.4242157923201731"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(seg_df['EncodedPixels']!='').mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "seg_df = seg_df[['Image_Label', 'EncodedPixels']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "seg_df.to_csv('../submission/1110_deeplabv3plus_resnet_512x768_v6_seed2001_with_clf.csv.gz', \n",
    "              index=False, compression='gzip')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Image_Label</th>\n",
       "      <th>EncodedPixels</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10055ff.jpg_Fish</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10055ff.jpg_Flower</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>10055ff.jpg_Gravel</td>\n",
       "      <td>1217 12 1560 2 1564 20 1599 2 1618 4 1899 3 19...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>10055ff.jpg_Sugar</td>\n",
       "      <td>23 100 266 37 311 5 360 127 592 81 706 135 939...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6e54759.jpg_Fish</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          Image_Label                                      EncodedPixels\n",
       "0    10055ff.jpg_Fish                                                   \n",
       "1  10055ff.jpg_Flower                                                   \n",
       "2  10055ff.jpg_Gravel  1217 12 1560 2 1564 20 1599 2 1618 4 1899 3 19...\n",
       "3   10055ff.jpg_Sugar  23 100 266 37 311 5 360 127 592 81 706 135 939...\n",
       "4    6e54759.jpg_Fish                                                   "
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "seg_df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## or select predicted empty with highest confidence, then remove them"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub_df.sort_values(['has_mask'], ascending=True, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-0.24907691001892096\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['efe623a.jpg_Fish',\n",
       " '8ab86a9.jpg_Fish',\n",
       " '9c0203c.jpg_Fish',\n",
       " '7130cc6.jpg_Fish',\n",
       " '971d047.jpg_Fish']"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cnt = 5000\n",
    "\n",
    "candidates = sub_df.iloc[:cnt, ]\n",
    "print(candidates.has_mask.max())\n",
    "candidates = candidates.Image_Label.to_list()\n",
    "candidates[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Image_Label</th>\n",
       "      <th>EncodedPixels</th>\n",
       "      <th>has_mask</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10055ff.jpg_Fish</td>\n",
       "      <td></td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10055ff.jpg_Flower</td>\n",
       "      <td></td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>10055ff.jpg_Gravel</td>\n",
       "      <td>3664 10 4005 20 4354 22 4704 23 5054 27 5404 2...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>10055ff.jpg_Sugar</td>\n",
       "      <td>1090 36 1139 1 1438 41 1486 9 1783 63 2132 65 ...</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6e54759.jpg_Fish</td>\n",
       "      <td></td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          Image_Label                                      EncodedPixels  \\\n",
       "0    10055ff.jpg_Fish                                                      \n",
       "1  10055ff.jpg_Flower                                                      \n",
       "2  10055ff.jpg_Gravel  3664 10 4005 20 4354 22 4704 23 5054 27 5404 2...   \n",
       "3   10055ff.jpg_Sugar  1090 36 1139 1 1438 41 1486 9 1783 63 2132 65 ...   \n",
       "4    6e54759.jpg_Fish                                                      \n",
       "\n",
       "   has_mask  \n",
       "0     False  \n",
       "1     False  \n",
       "2      True  \n",
       "3      True  \n",
       "4     False  "
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "seg_df = pd.read_csv('../submission/1110_deeplabv3plus_resnet_512x768_v11_seed2030.csv.gz').fillna('')\n",
    "seg_df['has_mask'] = seg_df.EncodedPixels!=''\n",
    "seg_df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>Image_Label</th>\n",
       "      <th>EncodedPixels</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>has_mask</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>False</th>\n",
       "      <td>4937</td>\n",
       "      <td>4937</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>True</th>\n",
       "      <td>63</td>\n",
       "      <td>63</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Image_Label EncodedPixels\n",
       "               count         count\n",
       "has_mask                          \n",
       "False           4937          4937\n",
       "True              63            63"
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "seg_df.loc[seg_df.Image_Label.isin(candidates), ].groupby(['has_mask']).agg(['count'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## be careful, now remove masks!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "63"
      ]
     },
     "execution_count": 131,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "to_remove = seg_df.loc[seg_df.Image_Label.isin(candidates), ].groupby(['has_mask']).get_group(1)['Image_Label'].to_list()\n",
    "len(to_remove)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.41752298539751215"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(seg_df['EncodedPixels']!='').mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "seg_df.loc[seg_df.Image_Label.isin(to_remove), 'EncodedPixels'] = ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.41326392644672794"
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(seg_df['EncodedPixels']!='').mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "seg_df = seg_df[['Image_Label', 'EncodedPixels']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "seg_df.to_csv('../submission/1108_deeplabv3plus_resnet_512x768_v6_seed2009_with_clf.csv.gz', index=False, compression='gzip')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**nonzero_classifier_efficientnet-b5_512x768_v2_seed2018.log**\n",
    "- remove 30, LB.663 to .664\n",
    "- remove 80, "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
